{
 "cells": [
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-17T14:23:08.491026Z",
     "start_time": "2025-09-17T14:23:06.078974Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import os\n",
    "import openai"
   ],
   "id": "dd2e4170edf67683",
   "outputs": [],
   "execution_count": 1
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-17T14:24:09.118389Z",
     "start_time": "2025-09-17T14:23:38.171248Z"
    }
   },
   "cell_type": "code",
   "source": "!curl https://www.ipcc.ch/report/ar6/wg2/downloads/report/IPCC_AR6_WGII_Chapter03.pdf --output IPCC_AR6_WGII_Chapter03.pdf",
   "id": "776942322a5e0d53",
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
      "                                 Dload  Upload   Total   Spent    Left  Speed\n",
      "\n",
      "  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0\n",
      "  0     0    0     0    0     0      0      0 --:--:--  0:00:30 --:--:--     0\n",
      "  0     0    0     0    0     0      0      0 --:--:--  0:00:30 --:--:--     0\n",
      "curl: (35) schannel: failed to receive handshake, SSL/TLS connection failed\n"
     ]
    }
   ],
   "execution_count": 2
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-17T14:24:30.195542Z",
     "start_time": "2025-09-17T14:24:24.562416Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from llama_index.core import SimpleDirectoryReader\n",
    "from llama_index.llms.openai import OpenAI\n",
    "from llama_index.core.evaluation import DatasetGenerator\n",
    "\n",
    "documents = SimpleDirectoryReader(\n",
    "    input_files=[\"IPCC_AR6_WGII_Chapter03.pdf\"]\n",
    ").load_data()\n",
    "\n",
    "# Shuffle the documents\n",
    "import random\n",
    "\n",
    "random.seed(42)\n",
    "random.shuffle(documents)\n",
    "\n",
    "gpt_35_llm = OpenAI(model=\"gpt-3.5-turbo\", temperature=0.3)"
   ],
   "id": "c5def9a9f530a614",
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "KeyboardInterrupt\n",
      "\n"
     ]
    }
   ],
   "execution_count": 3
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "question_gen_query = (\n",
    "    \"You are a Teacher/ Professor. Your task is to setup \"\n",
    "    \"a quiz/examination. Using the provided context, formulate \"\n",
    "    \"a single question that captures an important fact from the \"\n",
    "    \"context. Restrict the question to the context information provided.\"\n",
    ")\n",
    "\n",
    "dataset_generator = DatasetGenerator.from_documents(\n",
    "    documents[:50],\n",
    "    question_gen_query=question_gen_query,\n",
    "    llm=gpt_35_llm,\n",
    ")"
   ],
   "id": "c407e68db4d85119"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "# NOTE: this may take some time. Go grab a coffee!\n",
    "questions = dataset_generator.generate_questions_from_nodes(num=40)\n",
    "print(\"Generated \", len(questions), \" questions\")"
   ],
   "id": "cdbacbf357635acd"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "with open(\"train_questions.txt\", \"w\") as f:\n",
    "    for question in questions:\n",
    "        f.write(question + \"\\n\")"
   ],
   "id": "f9afa416fc9bc149"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "dataset_generator = DatasetGenerator.from_documents(\n",
    "    documents[\n",
    "        50:\n",
    "    ],  # since we generated ~1 question for 40 documents, we can skip the first 40\n",
    "    question_gen_query=question_gen_query,\n",
    "    llm=gpt_35_llm,\n",
    ")"
   ],
   "id": "ab94107d9d7ec98"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "# NOTE: this may take some time. Go grab a coffee!\n",
    "questions = dataset_generator.generate_questions_from_nodes(num=40)\n",
    "print(\"Generated \", len(questions), \" questions\")"
   ],
   "id": "b6c1173cede27a5b"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "with open(\"eval_questions.txt\", \"w\") as f:\n",
    "    for question in questions:\n",
    "        f.write(question + \"\\n\")"
   ],
   "id": "7fa572ae54fca0ea"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "questions = []\n",
    "with open(\"eval_questions.txt\", \"r\") as f:\n",
    "    for line in f:\n",
    "        questions.append(line.strip())"
   ],
   "id": "5a174ad99c89428e"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "from llama_index.core import VectorStoreIndex\n",
    "\n",
    "# limit the context window to 2048 tokens so that refine is used\n",
    "from llama_index.core import Settings\n",
    "\n",
    "Settings.context_window = 2048\n",
    "\n",
    "index = VectorStoreIndex.from_documents(\n",
    "    documents,\n",
    ")\n",
    "\n",
    "query_engine = index.as_query_engine(similarity_top_k=2, llm=gpt_35_llm)"
   ],
   "id": "1d4e265cf37a6bff"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "contexts = []\n",
    "answers = []\n",
    "\n",
    "for question in questions:\n",
    "    response = query_engine.query(question)\n",
    "    contexts.append([x.node.get_content() for x in response.source_nodes])\n",
    "    answers.append(str(response))"
   ],
   "id": "61359b78b01cf435"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "from datasets import Dataset\n",
    "from ragas import evaluate\n",
    "from ragas.metrics import answer_relevancy, faithfulness\n",
    "\n",
    "ds = Dataset.from_dict(\n",
    "    {\n",
    "        \"question\": questions,\n",
    "        \"answer\": answers,\n",
    "        \"contexts\": contexts,\n",
    "    }\n",
    ")\n",
    "\n",
    "result = evaluate(ds, [answer_relevancy, faithfulness])\n",
    "print(result)"
   ],
   "id": "159d57df6cccd3e4"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "from llama_index.llms.openai import OpenAI\n",
    "from llama_index.finetuning.callbacks import OpenAIFineTuningHandler\n",
    "from llama_index.core.callbacks import CallbackManager\n",
    "\n",
    "finetuning_handler = OpenAIFineTuningHandler()\n",
    "callback_manager = CallbackManager([finetuning_handler])\n",
    "\n",
    "llm = OpenAI(model=\"gpt-3.5-turbo\", temperature=0.3)\n",
    "llm.callback_manager = callback_manager"
   ],
   "id": "337477d33cb44a6a"
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
